'''
Created on 6/06/2009

@author: nick
'''

import re
from numpy import *

class traindata:
    '''
    @summary: Parses and formats the training data for our project, to be used with mlp.py
     to produce a working Muliti-Layer Perceptron.
    '''
    hasdict = False
    
    def getdictionary(self, datapath = ""):
        #Read the text file specified by datapath and generate training data from it
        #Open file for reading
        infile = False
        if datapath != "":
            infile = open(datapath)
        
        #Read data from file
        if infile:
            rawdata = infile.read()
            infile.seek(0)
            
            #split the string at spaces or new lines
            pattern = re.compile("\n|\t| |\\|/")
            rawdata = pattern.split(rawdata)
            
            #remove leading/trailing punctuation - we're not using it
            #we keep some punctuation and special symbols because they may constitute a new term (100% may be different to #100)
            #count number of occurrences of each word.
            worddict = {}
            outworddict = {}
            for word in rawdata:
                word = word.strip(" ?!.,'\";:()[]{}")
                if word != "":
                    #add new words, or increment word counter
                    if (word not in worddict and len(word) > 0):
                        worddict[word] = 1
                        if len(word) > 2:
                            outworddict[word] = 1
                    else:
                        worddict[word] += 1
                        if len(word) > 2:
                            outworddict[word] += 1

            #store for other methods to use
            traindata.rawdata = rawdata
            
            #generate word list limited to 300 based on the 300 most-occurring wordsin the scanned text
            worddict = sorted(worddict.iteritems(), key=lambda(k,v):v, reverse=True)
            worddict = worddict[:1000]
            worddict = dict(worddict)
            worddict = worddict.keys()
    
            #generate word list limited to 300 based on the 300 most-occurring wordsin the scanned text with a length greater than 2
            outworddict = sorted(outworddict.iteritems(), key=lambda(k,v):v, reverse=True)
            outworddict = outworddict[:300]
            outworddict = dict(outworddict)
            outworddict = outworddict.keys()
            
            traindata.worddict = worddict
            traindata.outworddict = outworddict

            #tell other methods a dict has been generated
            traindata.hasdict = True
            
            print "Input dictionary is {0} words, output dictionary is {1} words".format(len(worddict), len(outworddict))
            
            return worddict
        else:
            print "There was a problem opening the input file"
            
    
    getdictionary = classmethod(getdictionary)
    
    '''
    @summary: Generates test data sets plus the expected result.
    getdictionary must be called first, or the dictionary will not be defined,
    so the test set cannot be generated.
    '''
    def gettraindata(self):
        if traindata.hasdict:
            #init local vars
            worddict = traindata.worddict
            outworddict = traindata.outworddict
            rawdata = traindata.rawdata
            datalist = []
            words = []
            counter = 0
            for word in rawdata:
                word = word.strip(" ?!.,'\";:()[]{}")
                if counter < 2 and word in worddict:
                    #build a set of two words which are in the dictionary
                    words.append(worddict.index(word))
                    counter += 1
                elif counter == 2 and word in outworddict:
                    #have a triple, add to output
                    outwords = []
                    outwords.extend(words)
                    outwords.append(ord(word[0]))
                    outwords.append(ord(word[1]))
                    wordnum = outworddict.index(word)
                    wordbin = [0, 0, 0, 0, 0, 0, 0, 0, 0]#zeros(9)
                    #outwords.append(wordnum)
                    for i in range(8, -1, -1):
                        wordbin[i] = wordnum & 1
                        wordnum = wordnum >> 1
                    outwords.extend(wordbin)
                    datalist.append(outwords)
                    
                    if word in worddict:
                        #pop oldest word in the sequence
                        words.pop(0)
                        #add the current word
                        words.append(worddict.index(word))
                    else:
                        #reset everything
                        words = []
                        counter = 0
                else:
                    counter = 0
                    words = []
                    
            traindata.wordlist = datalist
            return datalist #False not implemented yet
        print "No dictionary available, cannot generate data. Call getdictionary first."
        return None

    gettraindata = classmethod(gettraindata)


    '''
    @summary: Returns a test data set to test the trained MLP network. 
    Takes a different data path for the training data, but relies on the same 
    dictionary as the gettrainingdata method.
    '''
    def gettestdata(self, datapath=""):
        if traindata.hasdict:
            #Read the text file specified by datapath and generate training data from it
            #Open file for reading
            infile = False
            if datapath != "":
                infile = open(datapath)
            
            #Read data from file
            if infile:
                rawdata = infile.read()
                infile.seek(0)
            
                #split the string at spaces or new lines
                pattern = re.compile("\n|\t| |\\|/")
                rawdata = pattern.split(rawdata)
                
                #init local vars
                worddict = traindata.worddict
                outworddict = traindata.outworddict
                datalist = []
                words = []
                counter = 0
                #cycle through all the words in the raw input
                for word in rawdata:
                    word = word.strip(" ?!.,'\";:()[]{}")
                    if counter < 2 and word in worddict:
                        #build a set of two words which are in the dictionary
                        words.append(worddict.index(word))
                        counter += 1
                    elif counter == 2 and word in outworddict:
                        #have a triple, add to output
                        outwords = []
                        outwords.extend(words)
                        outwords.append(ord(word[0]))
                        outwords.append(ord(word[1]))
                        wordnum = outworddict.index(word)
                        #outwords.append(wordnum)
                        wordbin = [0, 0, 0, 0, 0, 0, 0, 0, 0]#zeros(9)
                        for i in range(8, -1, -1):
                            wordbin[i] = wordnum & 1
                            wordnum = wordnum >> 1
                        outwords.extend(wordbin)
                        datalist.append(outwords)
                        
                        if word in worddict:
                            #pop oldest word in the sequence
                            words.pop(0)
                            #add the current word
                            words.append(worddict.index(word))
                        else:
                            #reset everything
                            words = []
                            counter = 0
                    else:
                        counter = 0
                        words = []
                    
            traindata.wordlist = datalist
            return datalist #False not implemented yet
        else:
            print "There was a problem opening the input file"
            return
        print "No dictionary available, cannot generate data. Call getdictionary first."
        
    gettestdata = classmethod(gettestdata)